# -*- coding:utf-8 -*-

# @Time : 2023/5/19 15:48
# @Author : 快乐的小猴子
# @Version : 
# @Function : 
import requests
from tongjiinfos.conf import settings
from tongjiinfos.conf.proxies import Proxies
from lxml import etree

class TongJiInfos():
    def send(self, url):
        ''' 发送请求，获取响应数据 '''
        # resp = requests.get(url, headers=settings.headers, proxies=Proxies().get_proxies()).content.decode()
        resp = requests.get(url, headers=settings.headers).content.decode()
        return resp

    def handle_resp(self, resp):
        '''处理响应数据，提取标题子页面的url'''
        url_new_list = []
        tree = etree.HTML(resp)
        topic_href_list = tree.xpath('//div[@class="list-group"]/div/ul/li/a/@href')
        for i in range(0, len(topic_href_list)):
            url_new = 'http://www.csrc.gov.cn/' + topic_href_list[i]
            url_new_list.append(url_new)
        return url_new_list

    def handle_son_resp(self, son_resp):
        tree = etree.HTML(son_resp)
        topic_names = tree.xpath('//div[@class="main-right fr common-list"]/div[1]/span/text()')[0].strip()
        title_names = tree.xpath('//div[@class="main-right fr common-list"]/ul/li/a/text()')
        news_urls = tree.xpath('//div[@class="main-right fr common-list"]/ul/li/a/@href')
        news_dates = tree.xpath('//div[@class="main-right fr common-list"]/ul/li/span/text()')
        # print(topic_names, title_names, news_urls, news_dates)
        return topic_names, title_names, news_urls, news_dates

    def main(self):
        resp = self.send(settings.main_url)
        url_new_list = self.handle_resp(resp)
        # for i in range(0, len(url_new_list)):
        for i in range(1, 2):
            # print(url_new_list[i])
            for j in range(1, 47):
                if j == 1:
                    url_new = url_new_list[i]
                    # print(j, url_new)
                else:
                    url_new = url_new_list[i].rsplit('.', 1)[0] + '_{}.'.format(j) + url_new_list[i].rsplit('.', 1)[1]
                print(j, url_new)
            # son_resp = self.send(url_new_list[i])
            # topic_names, title_names, news_urls, news_dates = self.handle_son_resp(son_resp)





tongjiinfos = TongJiInfos()
tongjiinfos.main()